{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "79327ba4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# key function\n",
    "from sklearn.metrics import mean_squared_error, r2_score\n",
    "from sklearn.model_selection import ParameterGrid\n",
    "\n",
    "# Loss Function\n",
    "# Huber objective function\n",
    "def huber(actual, predicted, xi):\n",
    "    actual, predicted = np.array(actual).flatten(), np.array(predicted).flatten()\n",
    "    resid = actual - predicted\n",
    "    huber_loss = np.where(np.abs(resid)<=xi, diff**2, 2*xi*np.abs(resid)-xi**2)\n",
    "    return np.mean(huber_loss)\n",
    "\n",
    "# Scoring Function\n",
    "# out-of-sample R squared\n",
    "def R_oos(actual, predicted):\n",
    "    actual, predicted = np.array(actual), np.array(predicted).flatten()\n",
    "    predicted = np.where(predicted<0,0,predicted)\n",
    "    return 1 - (np.dot((actual-predicted),(actual-predicted)))/(np.dot(actual,actual))\n",
    "\n",
    "# Validation Function\n",
    "def val_fun(model, params: dict, X_trn, y_trn, X_vld, y_vld, illustration=True, sleep=0, is_NN=False):\n",
    "    best_ros = None\n",
    "    lst_params = list(ParameterGrid(params))\n",
    "    for param in lst_params:\n",
    "        if best_ros == None:\n",
    "            if is_NN:\n",
    "                mod = model().set_params(**param).fit(X_trn, y_trn, X_vld, y_vld)\n",
    "            else:\n",
    "                mod = model().set_params(**param).fit(X_trn, y_trn)\n",
    "            best_mod = mod\n",
    "            y_pred = mod.predict(X_vld)\n",
    "            best_ros = R_oos(y_vld, y_pred)\n",
    "            best_param = param\n",
    "            if illustration:\n",
    "                print(f'Model with params: {param} finished.')\n",
    "                print(f'with out-of-sample R squared on validation set: {best_ros*100:.5f}%')\n",
    "                print('*'*60)\n",
    "        else:\n",
    "            time.sleep(sleep)\n",
    "            if is_NN:\n",
    "                mod = model().set_params(**param).fit(X_trn, y_trn, X_vld, y_vld)\n",
    "            else:\n",
    "                mod = model().set_params(**param).fit(X_trn, y_trn)\n",
    "            y_pred = mod.predict(X_vld)\n",
    "            ros = R_oos(y_vld, y_pred)\n",
    "            if illustration:\n",
    "                print(f'Model with params: {param} finished.')\n",
    "                print(f'with out-of-sample R squared on validation set: {ros*100:.5f}%')\n",
    "                print('*'*60)\n",
    "            if ros > best_ros:\n",
    "                best_ros = ros\n",
    "                best_mod = mod\n",
    "                best_param = param\n",
    "    if illustration:\n",
    "        print('\\n'+'#'*60)\n",
    "        print('Tuning process finished!!!')\n",
    "        print(f'The best setting is: {best_param}')\n",
    "        print(f'with R2oos {best_ros*100:.2f}% on validation set.')\n",
    "        print('#'*60)\n",
    "    return best_mod\n",
    "    \n",
    "\n",
    "# Pairwise Comparison\n",
    "# Diebold-Mariano test statistics\n",
    "\n",
    "# Evaluation Output\n",
    "def evaluate(actual, predicted, insample=False):\n",
    "    if insample:\n",
    "        print('*'*15+'In-Sample Metrics'+'*'*15)\n",
    "        print(f'The in-sample R2 is {r2_score(actual,predicted)*100:.2f}%')\n",
    "        print(f'The in-sample MSE is {mean_squared_error(actual,predicted):.3f}')\n",
    "    else:\n",
    "        print('*'*15+'Out-of-Sample Metrics'+'*'*15)\n",
    "        print(f'The out-of-sample R2 is {R_oos(actual,predicted)*100:.2f}%')\n",
    "        print(f'The out-of-sample MSE is {mean_squared_error(actual,predicted):.3f}')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "fhuan",
   "language": "python",
   "name": "fhuan"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
